def set_data(self, data):
"""Set the data dict with the data series required
for the generator.
Inputs:
data: dict - with keys matching those requested by
get_data_types.
"""
txmultigeneratormultisite.TxMultiGeneratorMultiSite.set_data(
self, data)
self.data = data[self.config['data_name']]
self.site_to_data = {}
if len(self.config['data_map_name']) > 0:
map_data = data[self.config['data_map_name']]
for i in range(0, map_data.shape[0]):
self.site_to_data[map_data[i, 0]] = map_data[i, 1]
if len(self.params_to_site) == 0:
# We have a data map, but no params to site, so assume
# all data are used and map 1:1 to this.
self.params_to_site = map_data[:, 0]
elif len(self.params_to_site) > 0:
# No data map is provided, but params_to_site is. If the
# lengths agree, map the site index list to the data 1:1.
if not (len(self.params_to_site) == self.data.shape[1]):
raise mureilexception.ConfigException(
'In model ' + self.config['section'] +
', no data map is provided, the data is width ' +
str(self.data.shape[1]) +
' and the provided params_to_site list is ' +
str(len(self.params_to_site)) +
' so no automatic mapping is possible.', {})
for i in range(len(self.params_to_site)):
self.site_to_data[self.params_to_site[i]] = i
else:
# No list of sites is provided. Just map to ordinal numbers.
self.params_to_site = range(self.data.shape[1])
for i in range(0, self.data.shape[1]):
self.site_to_data[i] = i
# Check that all of the values in site_to_data are within the
# self.data matrix.
max_data = self.data.shape[1]
for data_index in self.site_to_data.itervalues():
if (data_index < 0) or (data_index >= max_data):
raise mureilexception.ConfigException(
'data_index ' + str(data_index) +
' was requested by the model in section ' +
self.config['section'] +
' but the maximum index in the data array is ' +
str(max_data), {})
def set_startup_state(self, startup_data):
"""Set the startup state from the data provided. Sets
self.startup_state from this.
Inputs:
startup_data: An array of generators * 4:
[[site_index, capacity, build_date, decommissioning_period],
...]
"""
# Check if the startup data is empty. If so, just return.
if len(startup_data) == 0:
return
# Find out which build periods are covered.
startup_data = numpy.array(startup_data)
if not (len(startup_data.shape) == 2):
raise mureilexception.ConfigException(
'startup data array for module ' + self.config['section'] +
' is not rectangular.', {})
if not (startup_data.shape[1] == 4):
raise mureilexception.ConfigException(
'startup data array for module ' + self.config['section'] +
' shape ' + str(startup_data.shape) +
' but (n, 4) is required.', {})
self.extra_periods = map(
int, (list(set(startup_data[:, 2].tolist() + self.extra_periods))))
self.extra_periods.sort()
# And insert each existing generator into the starting state.
cap_list = self.startup_state['capacity']
hist_list = self.startup_state['history']
for i in range(startup_data.shape[0]):
site_index = int(startup_data[i, 0])
new_cap = startup_data[i, 1]
period = int(startup_data[i, 2])
decomm_date = int(startup_data[i, 3])
new_entry = (new_cap, period, decomm_date)
if decomm_date < self.run_periods[0]:
logger.warning(
'Model in section ' + self.config['section'] +
' adds startup capacity decommissioned at end of ' +
decomm_date + ' but the first run period is ' +
self.run_periods[0] +
' so it has been removed from the startup state.')
if site_index not in hist_list:
hist_list[site_index] = []
hist_list[site_index].append(new_entry)
else:
new_entry = (new_cap, period, decomm_date)
if site_index not in cap_list:
cap_list[site_index] = []
cap_list[site_index].append(new_entry)
def do_iteration(self):
if (not self.is_configured):
msg = 'do_iteration requested, but geneticalgorithm is not configured'
logger.critical(msg)
raise mureilexception.ConfigException(msg, {})
self.iteration_count += 1
self.population.mutate()
self.pop_score()
if self.iteration_count % 1 == 0:
try:
gene = iter(self.population.genes).next()
b_score = gene.score
bestgene = gene
except StopIteration:
b_score = Inf
bestgene = None
for gene in self.population.genes:
if gene.score > b_score:
bestgene = gene
b_score = gene.score
logger.debug('b_score = %f', b_score)
self.best_gene_data.append(
[bestgene.values[:], bestgene.score, self.iteration_count])
self.population.lemming()
self.population.breed()
self.decloner()
logger.debug('iteration: %d', self.iteration_count)
return None
def calculate_cost(self, state_handle, period, site_indices):
"""Calculate the cost of adding new transmission connections to the network,
from the list of active sites provided.
Inputs:
state_handle: The state_handle, as returned by get_startup_state.
site_indices: A list of sites requiring transmission connections,
which may include duplicates.
Outputs:
cost: The cost in $M for building the new transmission.
"""
curr_conf = self.period_configs[period]
uniq_sites = list(set(site_indices))
cost = 0.0
for site in uniq_sites:
try:
cost += self.dist_map[site] * curr_conf['cost_per_km']
except KeyError:
raise mureilexception.ConfigException(
'Site ' + str(site) + ' is not in transmission map.', {})
return cost
def run(self, extra_data=None):
start_time = time.time()
logger.critical('Run started at %s', time.ctime())
if (not self.is_configured):
msg = 'run requested, but txmultimastersimple is not configured'
logger.critical(msg)
raise mureilexception.ConfigException(msg, {})
try:
self.algorithm.prepare_run()
for i in range(self.config['iterations']):
self.algorithm.do_iteration()
if ((self.config['output_frequency'] > 0)
and ((i % self.config['output_frequency']) == 0)):
logger.info('Interim results at iteration %d', i)
self.output_results(iteration=i)
except mureilexception.AlgorithmException:
# Insert here something special to do if debugging
# such an exception is required.
# self.finalise will be called by the caller
raise
logger.critical('Run time: %.2f seconds', (time.time() - start_time))
results = self.output_results(iteration=self.config['iterations'],
final=True)
return results
def complete_configuration_pre_expand(self):
"""Complete the configuration prior to expanding the
period configs.
This implementation checks that the lifetime_yrs is a multiple
of time_period_yrs, and sets the startup state and params_to_site from the
configuration strings.
"""
time_period_yrs = self.config['time_period_yrs']
lifetime_yrs = self.config['lifetime_yrs']
error = None
if isinstance(lifetime_yrs, dict):
for value in lifetime_yrs.itervalues():
div = value / time_period_yrs
if not (float(int(div)) == div):
error = value
else:
div = lifetime_yrs / time_period_yrs
if not (float(int(div)) == div):
error = lifetime_yrs
if error is not None:
msg = ('In section ' + self.config['section'] + ', lifetime_yrs = ' +
str(error) + ' which is required to be a multiple of time_period_yrs of ' +
str(time_period_yrs))
raise mureilexception.ConfigException(msg, {})
# Set the startup state and the params to site from the configuration strings.
if self.config['startup_data_string'] is not None:
self.set_startup_state(self.config['startup_data_string'])
if len(self.config['params_to_site_data_string']) > 0:
self.params_to_site = self.config['params_to_site_data_string']
def solve_multiple_steps(self, market, multi_demand, multi_generation):
"""Solve the LP in the market object, for quantity values in a matrix for
demand and generation, which correspond to the bids and offer nodes when
the market object was created by build_optimisation.
Inputs:
market: a market optimisation created by build_optimisation
multi_demand: a matrix of quantities bid, corresponding to bid nodes
multi_generation: a matrix of quantities offered, corresponding to offer nodes
Outputs:
results: if success == True, a dict containing the following, or None:
scheduled_bids: a matrix of scheduled bids, corresponding to multi_demand
scheduled_offers: a matrix of scheduled offers, corresponding to multi_generation
solutions: The solutions object, for debug use.
Exceptions:
will raise SolverException if either the solver failed to reach an optimal
solution, or the solver was not run due to rejection by the
reject_outright_proportion check on total demand and total supply.
"""
logger.debug('multi demand')
logger.debug(multi_demand)
logger.debug('multi generation')
logger.debug(multi_generation)
solutions = []
if multi_demand.size[1] != multi_generation.size[1]:
msg = ('multi_demand.size[0] = ' + str(multi_demand.size[0]) +
', multi_demand.size[1] = ' + str(multi_demand.size[1]) +
', multi_generation.size[0] = ' +
str(multi_generation.size[0]) +
', multi_generation.size[1] = ' +
str(multi_generation.size[1]))
raise mureilexception.ConfigException(msg, {})
for j in range(multi_generation.size[1]):
# Check here that total demand isn't heaps more than total supply
tot_d = np.sum(multi_demand[:, j])
tot_g = np.sum(multi_generation[:, j])
if (tot_d / tot_g) > self.config['reject_outright_proportion']:
msg = 'Reject outright ' + str(tot_d / tot_g)
raise mureilexception.SolverException(msg,
{'prop': tot_d / tot_g})
self.update_program(market, multi_demand[:, j],
multi_generation[:, j])
this_sol = self.solve(market)
solutions.append(this_sol)
results = {}
schedules = cvx.matrix([s['x'].T for s in solutions]).T
results['scheduled_bids'] = self.scheduled_bids(market, schedules)
results['scheduled_offers'] = self.scheduled_offers(market, schedules)
return results, solutions
def complete_configuration(self):
# Check the pump_round_trip is <= 1
if self.config['pump_round_trip'] > 1:
msg = (
'BasicPumpedHydro requires pump_round_trip to be less than 1. '
+ ' Value = {:.3f}'.format(self.config['pump_round_trip']))
logger.critical(msg)
raise mureilexception.ConfigException(msg, {})
# Pre-calculate these for improved speed
# Instead of calculating explicitly the water that's pumped up, calculate
# the amount of electricity that's stored.
# Adjust here for the timestep - elec_res is in units of MW-timesteps.
# so 1 MWh = (60/timestep) MW-timesteps
self.elec_res = (1 / self.config['timestep_hrs']) * (
float(self.config['starting_level']) /
float(self.config['water_factor']))
self.elec_cap = (1 / self.config['timestep_hrs']) * (float(
self.config['dam_capacity']) / float(self.config['water_factor']))
self.pump_round_trip_recip = 1 / self.config['pump_round_trip']
self.is_configured = True
def complete_configuration_pre_expand(self):
"""Complete the configuration by setting the param-site map and pre-calculating the
fuel cost in $m/mwh.
"""
txmultigeneratormultisite.TxMultiGeneratorMultiSite.complete_configuration_pre_expand(self)
if isinstance(self.config['site_index'], dict):
msg = ('In model ' + self.config['model'] +
', the site_index parameter must not vary with time.')
raise mureilexception.ConfigException(msg, {})
self.params_to_site = numpy.array([self.config['site_index']])
fuel_price = self.config['fuel_price_mwh']
if isinstance(fuel_price, dict):
self.config['fuel_price_mwh_m'] = fpm = {}
for key, value in fuel_price:
fpm[key] = value / 1e6
else:
self.config['fuel_price_mwh_m'] = fuel_price / 1e6
def complete_configuration_post_expand(self):
"""Read in the grid information and site->node information.
"""
self.grid = grid_data_loader.Grid()
self.grid.load(self.config['grid_input_dir'],
self.config['grid_filenames'],
self.config['grid_remove_slack_nodes'])
self.nodes = []
for node in self.grid.nodes:
self.nodes.append(node['name'])
# and the site->node map - just a simple CSV read - would be neater
# to integrate this into the data system as it refers to generator indices.
import csv
# both site_to_node and site_connection_cost are dicts on site_index
self.site_to_node = {}
self.site_connection_cost = {}
with open(self.config['site_filename'], 'rU') as n:
reader = csv.reader(n)
for row in reader:
if reader.line_num == 1:
continue
else:
site_index = int(row[0])
node_name = row[1]
if node_name not in self.nodes:
msg = (
'When reading in transmission configuration, node '
+ node_name +
' is mapped to a site index, but is not in grid.')
raise mureilexception.ConfigException(msg, {})
self.site_to_node[site_index] = node_name
self.site_connection_cost[site_index] = float(row[2])
def run(self, extra_data=None):
start_time = time.time()
logger.critical('Run started at %s', time.ctime())
if (not self.is_configured):
msg = 'run requested, but GeTxMultiMaster is not configured'
logger.critical(msg)
raise mureilexception.ConfigException(msg, {})
# Read in the json data for generator capacity
period_lists, startup_lists, demand_settings = self.load_js(extra_data)
# Build up the results expected by the GE demo web code from the
# results.
all_years_out = {}
# Compute an annual total for generation
output_multiplier = (self.global_config['variable_cost_mult'] /
float(self.global_config['time_period_yrs']))
cuml_cost = 0.0
results = self.calc_list_cost(period_lists, startup_lists,
demand_settings)
for i in range(len(self.run_periods)):
period = self.run_periods[i]
all_years_out[str(period)] = period_out = {}
this_res = results['periods'][period]
# Output, in MWh
period_out['output'] = output_section = {}
# Cost, in $M
period_out['cost'] = cost_section = {}
# Total carbon emissions
period_out['co2_tonnes'] = 0.0
ts_demand = this_res['generators']['demand']['other']['ts_demand']
period_out['demand'] = '{:.2f}'.format(
abs(sum(ts_demand)) * self.global_config['timestep_hrs'] *
output_multiplier)
this_period_cost = 0.0
this_period_carbon = 0.0
for gen_type in this_res['generators']:
this_data = this_res['generators'][gen_type]
# Total output, in MWh per annum
output_section[gen_type] = '{:.2f}'.format(
this_data['total_supply_period'] /
self.global_config['time_period_yrs'])
# Total cost, per decade
cost_section[gen_type] = this_data['cost']
this_period_cost += this_data['cost']
# or as a string:
# cost_section[gen_type] = '{:.2f}'.format(this_data['cost'])
# Total carbon, per decade
this_period_carbon += sum(this_data['carbon_emissions_period'])
# Total cumulative cost, with discounting
# This assumes the costs are all incurred at the beginning of
# each period (a simplification)
period_out['period_cost'] = this_period_cost
cuml_cost += this_period_cost / (
(1 + (self.config['discount_rate'] / 100))
**(float(self.global_config['time_period_yrs']) * i))
period_out['discounted_cumulative_cost'] = cuml_cost
period_out['reliability'] = this_res['generators'][
'missed_supply']['other']['reliability']
period_out['co2_tonnes'] = this_period_carbon
return all_years_out
def load_js(self, json_data):
""" Input: JSON data structure with info on generators and demand management
at different time periods.
Output: None
Reads in the data and computes the params for each time period.
"""
generators = json.loads(json_data)['selections']['generators']
## Only coal, gas, wind and solar are handled, and only one of each.
## hydro is awaiting a rainfall-based model.
self.total_params = {}
self.inc_params = {}
gen_total_table = {}
gen_inc_table = {}
year_list = self.config['year_list']
gen_type_list = ['coal', 'gas', 'wind', 'solar']
gen_param_counts = {}
# Initialise the tables of capacity
for gen_type in gen_type_list:
gen_param_counts[gen_type] = self.gen_list[
gen_type].get_param_count()
gen_total_table[gen_type] = numpy.zeros(
(len(self.config['year_list']), gen_param_counts[gen_type]))
gen_inc_table[gen_type] = numpy.zeros(
(len(self.config['year_list']), gen_param_counts[gen_type]))
# Fill in the tables of capacity
for gen in generators:
gen_type = gen['type']
if gen_type not in gen_type_list:
msg = 'Generator ' + str(gen_type) + ' ignored'
logger.warning(msg)
else:
this_total_table = gen_total_table[gen_type]
this_inc_table = gen_inc_table[gen_type]
loc_index = self.find_loc_index(gen)
if (loc_index >= gen_param_counts[gen_type]):
msg = ('Generator ' + gen['id'] + ' looked up index as ' +
str(loc_index) + ' but the ' + gen_type +
' has data for ' + str(gen_param_counts[gen_type]) +
' sites.')
raise mureilexception.ConfigException(msg, {})
# build date could be specified as earlier, so capex is not paid.
build_index = numpy.where(
numpy.array(year_list) == str(gen['decade']))
if len(build_index[0] > 0):
build_index = build_index[0][0]
else:
build_index = -1
decommission_index = numpy.where(
numpy.array(year_list) == str(gen['decomission']))
if len(decommission_index[0] > 0):
decommission_index = decommission_index[0][0]
else:
decommission_index = len(year_list) - 1
# accumulate new capacity in the incremental list
if build_index >= 0:
this_inc_table[build_index][loc_index] += gen['capacity']
# and add the new capacity to the total across all years until decommissioning
start_fill = build_index
if (build_index == -1):
start_fill = 0
for i in range(start_fill, decommission_index + 1):
this_total_table[i][loc_index] += gen['capacity']
# Convert the tables of capacity to params for the sim
for i in range(0, len(year_list)):
this_total_params = numpy.zeros(self.param_count)
this_inc_params = numpy.zeros(self.param_count)
for gen_type in ['coal', 'wind', 'solar', 'gas']:
param_ptr = self.gen_params[gen_type]
if (param_ptr[0] < param_ptr[1]) and (gen_type
in gen_total_table):
this_total_params[param_ptr[0]:param_ptr[1]] = (
gen_total_table[gen_type][i])
this_inc_params[param_ptr[0]:param_ptr[1]] = (
gen_inc_table[gen_type][i])
self.total_params[str(year_list[i])] = this_total_params
self.inc_params[str(year_list[i])] = this_inc_params
self.demand_settings = json.loads(json_data)['selections']['demand']
def gene_test_undef(self, dummy):
raise mureilexception.ConfigException(
'gene_test_callback not set in geneticalgorithm.py',
__name__ + '.gene_test_undef', [])
def set_config(self, full_config, extra_data):
# Master explicitly does not copy in the global variables. It is too confusing
# to combine those with flags, defaults and values defined in the config files.
self.load_initial_config(full_config['Master'])
# Get the global variables
mureilbuilder.check_section_exists(full_config, self.config['global'])
if 'model' not in full_config[self.config['global']]:
full_config[self.config['global']][
'model'] = 'tools.globalconfig.GlobalBase'
self.global_calc = mureilbuilder.create_instance(
full_config, None, self.config['global'],
mureilbase.ConfigurableInterface)
self.global_config = self.global_calc.get_config()
# Now check the dispatch_order, to get a list of the generators
for gen in self.config['dispatch_order']:
self.config_spec += [(gen, None, None)]
self.update_from_config_spec()
self.check_config()
self.dispatch_order = self.config['dispatch_order']
# Set up the data class and get the data, and compute the global parameters
self.data = mureilbuilder.create_instance(
full_config, self.global_config, self.config['data'],
mureilbase.DataSinglePassInterface)
self.global_calc.update_config(
{'data_ts_length': self.data.get_ts_length()})
self.global_calc.post_data_global_calcs()
self.global_config = self.global_calc.get_config()
# Instantiate the transmission model
if self.config['transmission'] in full_config:
self.transmission = mureilbuilder.create_instance(
full_config, self.global_config, self.config['transmission'],
configurablebase.ConfigurableMultiBase,
self.config['run_periods'])
mureilbuilder.supply_single_pass_data(self.transmission, self.data,
self.config['transmission'])
else:
self.transmission = None
# Instantiate the generator objects, set their data
self.gen_list = {}
for i in range(len(self.dispatch_order)):
gen_type = self.dispatch_order[i]
# Build the generator instances
gen = mureilbuilder.create_instance(
full_config, self.global_config, self.config[gen_type],
txmultigeneratorbase.TxMultiGeneratorBase,
self.config['run_periods'])
self.gen_list[gen_type] = gen
# Supply data as requested by the generator
mureilbuilder.supply_single_pass_data(gen, self.data, gen_type)
# Check that run_periods increases by time_period_yrs
self.run_periods = self.config['run_periods']
if len(self.run_periods) > 1:
run_period_diffs = numpy.diff(self.run_periods)
if (not (min(run_period_diffs)
== self.global_config['time_period_yrs'])
or not (max(run_period_diffs)
== self.global_config['time_period_yrs'])):
raise mureilexception.ConfigException(
'run_periods must be separated by time_period_yrs', {})
self.period_count = len(self.run_periods)
self.is_configured = True
def set_config(self, full_config, extra_data):
# Master explicitly does not copy in the global variables. It is too confusing
# to combine those with flags, defaults and values defined in the config files.
self.load_initial_config(full_config['Master'])
# Get the global variables
mureilbuilder.check_section_exists(full_config, self.config['global'])
if 'model' not in full_config[self.config['global']]:
full_config[self.config['global']][
'model'] = 'tools.globalconfig.GlobalBase'
self.global_calc = mureilbuilder.create_instance(
full_config, None, self.config['global'],
mureilbase.ConfigurableInterface)
self.global_config = self.global_calc.get_config()
# Now check the dispatch_order, to get a list of the generators
for gen in self.config['dispatch_order']:
self.config_spec += [(gen, None, None)]
self.update_from_config_spec()
self.check_config()
self.dispatch_order = self.config['dispatch_order']
# Set up the data class and get the data, and compute the global parameters
self.data = mureilbuilder.create_instance(
full_config, self.global_config, self.config['data'],
mureilbase.DataSinglePassInterface)
self.global_calc.update_config(
{'data_ts_length': self.data.get_ts_length()})
self.global_calc.post_data_global_calcs()
self.global_config = self.global_calc.get_config()
# Instantiate the generator objects, set their data, determine their param requirements
param_count = 0
self.gen_list = {}
self.gen_params = {}
start_values_min = []
start_values_max = []
for i in range(len(self.dispatch_order)):
gen_type = self.dispatch_order[i]
# Build the generator instances
gen = mureilbuilder.create_instance(
full_config, self.global_config, self.config[gen_type],
singlepassgenerator.SinglePassGeneratorBase)
self.gen_list[gen_type] = gen
# Supply data as requested by the generator
mureilbuilder.supply_single_pass_data(gen, self.data, gen_type)
# Determine how many parameters this generator requires and
# allocate the slots in the params list
params_req = gen.get_param_count()
if (params_req == 0):
self.gen_params[gen_type] = (0, 0)
else:
self.gen_params[gen_type] = (param_count,
param_count + params_req)
(starts_min, starts_max) = gen.get_param_starts()
if len(starts_min) == 0:
start_values_min += (
(np.ones(params_req) *
self.global_config['min_param_val']).tolist())
else:
start_values_min += starts_min
if len(starts_max) == 0:
start_values_max += (
(np.ones(params_req) *
self.global_config['max_param_val']).tolist())
else:
start_values_max += starts_max
param_count += params_req
self.param_count = param_count
# Check if 'extra_data' has been provided, as a full gene to start at.
# extra_data needs to be a dict with entry 'start_gene' that is a list
# of integer values the same length as param_count.
if extra_data is not None:
if 'start_gene' in extra_data:
if not (len(start_values_min) == param_count):
msg = (
'extra_data of start_gene passed to simplemureilmaster. '
+ 'Length expected = {:d}, found = {:d}'.format(
param_count, len(start_values_min)))
raise mureilexception.ConfigException(msg, {})
else:
start_values_min = extra_data['start_gene']
start_values_max = extra_data['start_gene']
# Instantiate the genetic algorithm
mureilbuilder.check_section_exists(full_config,
self.config['algorithm'])
algorithm_config = full_config[self.config['algorithm']]
algorithm_config['min_len'] = algorithm_config['max_len'] = param_count
algorithm_config['start_values_min'] = start_values_min
algorithm_config['start_values_max'] = start_values_max
algorithm_config['gene_test_callback'] = self.gene_test
self.algorithm = mureilbuilder.create_instance(
full_config, self.global_config, self.config['algorithm'],
mureilbase.ConfigurableInterface)
self.is_configured = True
def get_timeseries(self, ts_name):
try:
return self.data[ts_name]
except:
msg = 'Timeseries ' + str(ts_name) + ' requested, but not available.'
raise mureilexception.ConfigException(msg, {})
def complete_configuration(self):
self.data = {}
for list_type in [
'ts_float_list', 'ts_int_list', 'other_float_list',
'other_int_list'
]:
for series_name in self.config[list_type]:
infile = self.config['dir'] + self.config[series_name +
'_file']
try:
f = nc.NetCDFFile(infile)
except:
msg = ('File ' + infile + ' for data series ' +
series_name + ' was not opened.')
raise mureilexception.ConfigException(msg, {})
try:
vbl = f.variables[self.config[series_name + '_vbl']]
except:
msg = ('Variable ' + self.config[series_name + '_vbl'] +
' not found in file ' + infile)
raise mureilexception.ConfigException(msg, {})
dims = len(vbl.shape)
if (dims == 1):
temp = vbl[:]
elif (dims == 2):
temp = vbl[:, :]
else:
msg = 'Data series ' + series_name + ' has more than 2 dimensions, so is not handled.'
raise mureilexception.ConfigException(msg, {})
if 'float' in list_type:
if not mureiltypes.check_ndarray_float(temp, True):
self.data[series_name] = numpy.array(temp, dtype=float)
else:
self.data[series_name] = temp
else:
if not mureiltypes.check_ndarray_int(temp, True):
self.data[series_name] = numpy.array(temp, dtype=int)
else:
self.data[series_name] = temp
for list_type in ['ts_csv_list']:
for series_name in self.config[list_type]:
infile = self.config['dir'] + self.config[series_name +
'_file']
temp = []
try:
with open(infile, 'rU') as n:
reader = csv.reader(n)
for row in reader:
if reader.line_num == 1:
self.data[series_name + '_hdr'] = row[1:]
else:
if reader.line_num == 2:
temp = [map(float, (row[1:]))]
else:
temp.append(map(float, (row[1:])))
self.data[series_name] = numpy.array(temp, dtype=float)
if not mureiltypes.check_ndarray_float(temp, True):
self.data[series_name] = numpy.array(temp, dtype=float)
else:
self.data[series_name] = temp
except:
msg = ('File ' + infile + ' for data series ' +
series_name +
' was not opened or had an error in reading.')
raise mureilexception.ConfigException(msg, {})
# Now apply the NaN filter to the ts lists, but note that the integer
# ones are not identified as nan.
all_ts = self.config['ts_float_list'] + self.config[
'ts_int_list'] + self.config['ts_csv_list']
if len(all_ts) == 0:
self.ts_length = 0
logger.warning('No timeseries data defined')
else:
self.ts_length = self.data[all_ts[0]].shape[0]
# Start with an array of 'False'
nan_acc = (numpy.ones(self.ts_length) == 0)
# Accumulate 'True' entries in nan_acc where NaN found in timeseries
for ts_name in all_ts:
ts_nan = numpy.isnan(self.data[ts_name])
if ts_nan.ndim > 1:
ts_nan = ts_nan.any(1)
# Check all the timeseries are the same length
if not (len(ts_nan) == self.ts_length):
msg = ('Data series ' + ts_name +
' is length {:d}, not matching {:d} of '.format(
len(ts_nan), self.ts_length) + all_ts[0])
raise mureilexception.ConfigException(msg, {})
nan_acc = numpy.logical_or(nan_acc, ts_nan)
# Clean up the timeseries using slices
nan_acc = numpy.logical_not(nan_acc)
for ts_name in all_ts:
if self.data[ts_name].ndim == 1:
self.data[ts_name] = self.data[ts_name][nan_acc]
else:
self.data[ts_name] = self.data[ts_name][nan_acc, :]
self.ts_length = self.data[all_ts[0]].shape[0]
self.is_configured = True
return None
def get_ts_length(self):
try:
return self.ts_length
except:
msg = 'Data ts length requested, but ts_length not available'
raise mureilexception.ConfigException(msg, {})
def run(self, extra_data):
if (not self.is_configured):
msg = 'run requested, but GeMureilMaster is not configured'
logger.critical(msg)
raise mureilexception.ConfigException(msg, {})
# Read in the json data for generator capacity
self.load_js(extra_data)
all_years_out = {}
# Compute an annual total for generation
output_multiplier = (self.global_config['variable_cost_mult'] /
float(self.global_config['time_period_yrs']))
cuml_cost = 0.0
for year_index in range(len(self.config['year_list'])):
## MG - this is a hack. The config should be set with all
## of the values at the start, and then be passed the year,
## not have them updated each time. This is ok here as it's
## only evaluated once anyway.
results = self.evaluate_results(year_index)
year = self.config['year_list'][year_index]
# print results['gen_desc']
all_years_out[str(year)] = year_out = {}
# Output, in MWh
year_out['output'] = output_section = {}
# Cost, in $M
year_out['cost'] = cost_section = {}
# Total carbon emissions
year_out['co2_tonnes'] = 0.0
# Total demand, in MWh per annum
for generator_type, value in results['other'].iteritems():
if value is not None:
if 'ts_demand' in value:
year_out['demand'] = '{:.2f}'.format(
abs(sum(value['ts_demand'])) *
self.global_config['timestep_hrs'] *
output_multiplier)
# Total output, in MWh per annum
for gen_type, vals in results['output'].iteritems():
output_section[gen_type] = '{:.2f}'.format(
sum(vals) * self.global_config['timestep_hrs'] *
output_multiplier)
# Total cost, per decade
this_period_cost = 0.0
for gen_type, value in results['cost'].iteritems():
cost_section[gen_type] = value
this_period_cost += value
# or as a string:
# cost_section[generator_type] = '{:.2f}'.format(value)
# Total cumulative cost, with discounting
# This assumes the costs are all incurred at the beginning of
# each period (a simplification)
year_out['period_cost'] = this_period_cost
cuml_cost += this_period_cost / (
(1 + (self.config['discount_rate'] / 100))
**(float(self.global_config['time_period_yrs']) * year_index))
year_out['discounted_cumulative_cost'] = cuml_cost
for gen_type, value in results['other'].iteritems():
if value is not None:
if 'reliability' in value:
year_out['reliability'] = value['reliability']
if 'carbon' in value:
year_out['co2_tonnes'] += value['carbon']
if 'reliability' not in year_out:
year_out['reliability'] = 100
return all_years_out
def set_config(self, full_config, extra_data):
# Master explicitly does not copy in the global variables. It is too confusing
# to combine those with flags, defaults and values defined in the config files.
self.load_initial_config(full_config['Master'])
# Get the global variables
mureilbuilder.check_section_exists(full_config, self.config['global'])
if 'model' not in full_config[self.config['global']]:
full_config[self.config['global']][
'model'] = 'tools.globalconfig.GlobalBase'
self.global_calc = mureilbuilder.create_instance(
full_config, None, self.config['global'],
mureilbase.ConfigurableInterface)
self.global_config = self.global_calc.get_config()
# Now check the dispatch_order, to get a list of the generators
for gen in self.config['dispatch_order']:
self.config_spec += [(gen, None, None)]
self.update_from_config_spec()
self.check_config()
self.dispatch_order = self.config['dispatch_order']
# Set up the data class and get the data, and compute the global parameters
self.data = mureilbuilder.create_instance(
full_config, self.global_config, self.config['data'],
mureilbase.DataSinglePassInterface)
self.global_calc.update_config(
{'data_ts_length': self.data.get_ts_length()})
self.global_calc.post_data_global_calcs()
self.global_config = self.global_calc.get_config()
# Instantiate the transmission model
if self.config['transmission'] in full_config:
self.transmission = mureilbuilder.create_instance(
full_config, self.global_config, self.config['transmission'],
configurablebase.ConfigurableMultiBase,
self.config['run_periods'])
mureilbuilder.supply_single_pass_data(self.transmission, self.data,
self.config['transmission'])
else:
self.transmission = None
# Instantiate the generator objects, set their data, determine their param requirements
param_count = 0
self.gen_list = {}
self.gen_params = {}
run_period_len = len(self.config['run_periods'])
start_values_min = numpy.array([[]]).reshape(run_period_len, 0)
start_values_max = numpy.array([[]]).reshape(run_period_len, 0)
for i in range(len(self.dispatch_order)):
gen_type = self.dispatch_order[i]
# Build the generator instances
gen = mureilbuilder.create_instance(
full_config, self.global_config, self.config[gen_type],
txmultigeneratorbase.TxMultiGeneratorBase,
self.config['run_periods'])
self.gen_list[gen_type] = gen
# Supply data as requested by the generator
mureilbuilder.supply_single_pass_data(gen, self.data, gen_type)
# Determine how many parameters this generator requires and
# allocate the slots in the params list
params_req = gen.get_param_count()
if (params_req == 0):
self.gen_params[gen_type] = (0, 0)
else:
self.gen_params[gen_type] = (param_count,
param_count + params_req)
start_values_min, start_values_max = mureilbuilder.add_param_starts(
gen.get_param_starts(), params_req, self.global_config,
run_period_len, start_values_min, start_values_max)
param_count += params_req
start_values_min = start_values_min.reshape(run_period_len *
param_count)
start_values_max = start_values_max.reshape(run_period_len *
param_count)
self.param_count = param_count
# Check that run_periods increases by time_period_yrs
self.run_periods = self.config['run_periods']
if len(self.run_periods) > 1:
run_period_diffs = numpy.diff(self.run_periods)
if (not (min(run_period_diffs)
== self.global_config['time_period_yrs'])
or not (max(run_period_diffs)
== self.global_config['time_period_yrs'])):
raise mureilexception.ConfigException(
'run_periods must be separated by time_period_yrs', {})
self.period_count = len(self.run_periods)
self.total_param_count = param_count * self.period_count
# Check if 'extra_data' has been provided, as a full gene to start at.
# extra_data needs to be a dict with entry 'start_gene' that is a list
# of integer values the same length as param_count.
if extra_data is not None:
if 'start_gene' in extra_data:
if not (len(start_values_min) == self.total_param_count):
msg = (
'extra_data of start_gene passed to txmultimastersimple. '
+ 'Length expected = {:d}, found = {:d}'.format(
self.total_param_count, len(start_values_min)))
raise mureilexception.ConfigException(msg, {})
start_values_min = extra_data['start_gene']
start_values_max = extra_data['start_gene']
# Instantiate the genetic algorithm
mureilbuilder.check_section_exists(full_config,
self.config['algorithm'])
algorithm_config = full_config[self.config['algorithm']]
algorithm_config['min_len'] = algorithm_config[
'max_len'] = self.total_param_count
algorithm_config['start_values_min'] = start_values_min
algorithm_config['start_values_max'] = start_values_max
algorithm_config['gene_test_callback'] = self.gene_test
self.algorithm = mureilbuilder.create_instance(
full_config, self.global_config, self.config['algorithm'],
mureilbase.ConfigurableInterface)
self.is_configured = True
def set_config(self, full_config, extra_data):
# Master explicitly does not copy in the global variables. It is too confusing
# to combine those with flags, defaults and values defined in the config files.
self.load_initial_config(full_config['Master'])
# Get the global variables
mureilbuilder.check_section_exists(full_config, self.config['global'])
if 'model' not in full_config[self.config['global']]:
full_config[self.config['global']][
'model'] = 'tools.globalconfig.GlobalBase'
self.global_calc = mureilbuilder.create_instance(
full_config, None, self.config['global'],
mureilbase.ConfigurableInterface)
self.global_config = self.global_calc.get_config()
# Now check the list of the generators
for gen in self.config['generators']:
self.config_spec += [(gen, None, None)]
self.update_from_config_spec()
self.check_config()
self.generators = self.config['generators']
# Set up the data class and get the data, and compute the global parameters
self.data = mureilbuilder.create_instance(
full_config, self.global_config, self.config['data'],
mureilbase.DataSinglePassInterface)
self.global_calc.update_config(
{'data_ts_length': self.data.get_ts_length()})
self.global_calc.post_data_global_calcs()
self.global_config = self.global_calc.get_config()
## The master here takes 'global' variables, if needed and available, to get at the carbon_price_m,
## variable_cost_mult and time_scale_up_mult values, which it then expands to all time periods,
## for use later.
for param_name in [
'carbon_price_m', 'variable_cost_mult', 'time_scale_up_mult'
]:
if (param_name not in self.config) and (param_name
in self.global_config):
self.config[param_name] = self.global_config[param_name]
self.config_spec += [('carbon_price_m', float, None),
('variable_cost_mult', float, None),
('time_scale_up_mult', float, None)]
self.check_config()
self.expand_config(self.config['run_periods'])
# Now instantiate the demand model
self.demand = mureilbuilder.create_instance(
full_config, self.global_config, self.config['demand'],
configurablebase.ConfigurableMultiBase, self.config['run_periods'])
# Supply data to the demand model
mureilbuilder.supply_single_pass_data(self.demand, self.data, 'demand')
# And instantiate the transmission model
self.transmission = mureilbuilder.create_instance(
full_config, self.global_config, self.config['transmission'],
configurablebase.ConfigurableMultiBase, self.config['run_periods'])
mureilbuilder.check_subclass(
self.transmission, interfacesflowmaster.InterfaceTransmission)
# Instantiate the generator objects, set their data, determine their param requirements,
# and separate by dispatch type.
param_count = 0
# gen_list, gen_params are indexed by position in self.generators
self.gen_list = [None] * len(self.generators)
self.gen_params = [None] * len(self.generators)
self.semisch_list = []
self.instant_list = []
self.ramp_list = []
run_period_len = len(self.config['run_periods'])
start_values_min = numpy.array([[]]).reshape(run_period_len, 0)
start_values_max = numpy.array([[]]).reshape(run_period_len, 0)
for i in range(len(self.generators)):
gen_type = self.generators[i]
# Build the generator instances
gen = mureilbuilder.create_instance(
full_config, self.global_config, self.config[gen_type],
txmultigeneratorbase.TxMultiGeneratorBase,
self.config['run_periods'])
self.gen_list[i] = gen
gen_details = gen.get_details()
gen_dispatch = gen_details['dispatch']
if (gen_dispatch == 'semischeduled'):
self.semisch_list.append(i)
mureilbuilder.check_subclass(
gen, interfacesflowmaster.InterfaceSemiScheduledDispatch)
elif (gen_dispatch == 'instant'):
self.instant_list.append(i)
mureilbuilder.check_subclass(
gen, interfacesflowmaster.InterfaceInstantDispatch)
elif (gen_dispatch == 'ramp'):
self.ramp_list.append(i)
mureilbuilder.check_subclass(
gen, interfacesflowmaster.InterfaceRampDispatch)
msg = ("Generator " + gen_type +
" has dispatch type ramp, which is not yet implemented")
raise mureilexception.ConfigException(msg, {})
else:
msg = ("Generator " + gen_type + " has dispatch type " +
gen_dispatch +
" which is not one of semischeduled, instant or ramp.")
raise mureilexception.ConfigException(msg, {})
# Supply data as requested by the generator
mureilbuilder.supply_single_pass_data(gen, self.data, gen_type)
# Determine how many parameters this generator requires and
# allocate the slots in the params list
params_req = gen.get_param_count()
if (params_req == 0):
self.gen_params[i] = (0, 0)
else:
self.gen_params[i] = (param_count, param_count + params_req)
start_values_min, start_values_max = mureilbuilder.add_param_starts(
gen.get_param_starts(), params_req, self.global_config,
run_period_len, start_values_min, start_values_max)
param_count += params_req
start_values_min = start_values_min.reshape(run_period_len *
param_count)
start_values_max = start_values_max.reshape(run_period_len *
param_count)
self.param_count = param_count
# Check that run_periods increases by time_period_yrs
self.run_periods = self.config['run_periods']
if len(self.run_periods) > 1:
run_period_diffs = numpy.diff(self.run_periods)
if (not (min(run_period_diffs)
== self.global_config['time_period_yrs'])
or not (max(run_period_diffs)
== self.global_config['time_period_yrs'])):
raise mureilexception.ConfigException(
'run_periods must be separated by time_period_yrs', {})
self.period_count = len(self.run_periods)
self.total_param_count = param_count * self.period_count
# Check if 'extra_data' has been provided, as a full gene to start at.
# extra_data needs to be a dict with entry 'start_gene' that is a list
# of integer values the same length as param_count.
if extra_data is not None:
if 'start_gene' in extra_data:
if not (len(start_values_min) == self.total_param_count):
msg = (
'extra_data of start_gene passed to txmultimasterflow. '
+ 'Length expected = {:d}, found = {:d}'.format(
self.total_param_count, len(start_values_min)))
raise mureilexception.ConfigException(msg, {})
start_values_min = extra_data['start_gene']
start_values_max = extra_data['start_gene']
# Instantiate the market solver
self.market_solver = mureilbuilder.create_instance(
full_config, self.global_config, self.config['market_solver'],
configurablebase.ConfigurableMultiBase, self.config['run_periods'])
# Instantiate the genetic algorithm
mureilbuilder.check_section_exists(full_config,
self.config['algorithm'])
algorithm_config = full_config[self.config['algorithm']]
algorithm_config['min_len'] = algorithm_config[
'max_len'] = self.total_param_count
algorithm_config['start_values_min'] = start_values_min
algorithm_config['start_values_max'] = start_values_max
algorithm_config['gene_test_callback'] = self.gene_test
self.algorithm = mureilbuilder.create_instance(
full_config, self.global_config, self.config['algorithm'],
mureilbase.ConfigurableInterface)
self.is_configured = True
