def test_lazy_loop_calculator_cls():
"""Test the lazy loop calculator class."""
calc = {'formula': 'pythagorian_thm',
'args': {'data': {'adjacent': 'a', 'opposite': 'b'}, 'outputs': {}},
'returns': ['c']}
formula_reg = FormulaRegistry()
formula_reg.register(
{'pythagorian_thm': UREG.wraps(*PYTHAGOREAN_UNITS)(f_pythagorian_thm)},
args={'pythagorian_thm': ['adjacent', 'opposite']},
units={'pythagorian_thm': PYTHAGOREAN_UNITS},
isconstant={'pythagorian_thm': None}
)
data_reg = DataRegistry()
data_reg.register(
{'a': [3., 5., 7., 9., 11.] * UREG('cm'),
'b': [4., 12., 24., 40., 60.] * UREG('cm')},
uncertainty=None,
variance=None,
isconstant={'a': True, 'b': True}
)
out_reg = OutputRegistry()
out_reg.register({'c': np.zeros(5) * UREG.m})
# repeat args are listed as formula names, not data reg names!
calculator = LazyLoopingCalculator(repeat_args=['adjacent', 'opposite'])
calculator.calculate(calc, formula_reg, data_reg, out_reg)
assert np.allclose(out_reg['c'].m, PYTHAGOREAN_TRIPLES) # check magnitudes
assert out_reg['c'].u == UREG.m # output units are meters
return out_reg
def _apply_units_to_numpy_data_readers(parameters, data):
"""
Apply units to data originally loaded by :class:`NumPyLoadTxtReader` or
:class:`NumPyGenFromTxtReader`.
:param parameters: Dictionary of data source parameters read from JSON
file.
:type parameters: dict
:param data: Dictionary of data read
"""
# apply header units
header_param = parameters.get('header') # default is None
# check for headers
if header_param:
fields = header_param['fields'] # header fields
# dictionary of header field parameters
header_fields = {field[0]: field[1:] for field in fields}
# loop over fieldnames
for k, val in header_fields.iteritems():
# check for units in header field parameters
if len(val) > 1:
data[k] *= UREG(str(val[1])) # apply units
# apply other data units
data_units = parameters['data'].get('units') # default is None
if data_units:
for k, val in data_units.iteritems():
data[k] *= UREG(str(val)) # apply units
return data
def test_hdf5_reader():
"""
Test :class:`simkit.contrib.readers.HDF5Reader`
:return: readers and data
"""
setup_hdf5_test_data()
# test 1: load data from hdf5 dataset array by node
params = {
'GHI': {
'units': 'W/m**2',
'extras': {
'node': '/data/GHI'
}
},
'DNI': {
'units': 'W/m**2',
'extras': {
'node': '/data/DNI'
}
},
'Tdry': {
'units': 'degC',
'extras': {
'node': '/data/Tdry'
}
}
}
reader1 = HDF5Reader(params)
assert isinstance(reader1, DataReader)
data1 = reader1.load_data(H5TEST1)
assert np.allclose(data1['GHI'], H5TABLE['GlobalHorizontalRadiation'])
assert data1['GHI'].units == UREG('W/m**2')
assert np.allclose(data1['DNI'], H5TABLE['DirectNormalRadiation'])
assert data1['DNI'].units == UREG('W/m**2')
assert np.allclose(data1['Tdry'], H5TABLE['DryBulbTemperature'])
assert data1['Tdry'].units == UREG.degC
# test 2: load data from hdf5 dataset table by node and member name
params['GHI']['extras']['node'] = 'data'
params['GHI']['extras']['member'] = 'GlobalHorizontalRadiation'
params['DNI']['extras']['node'] = 'data'
params['DNI']['extras']['member'] = 'DirectNormalRadiation'
params['Tdry']['extras']['node'] = 'data'
params['Tdry']['extras']['member'] = 'DryBulbTemperature'
reader2 = HDF5Reader(params)
assert isinstance(reader1, DataReader)
data2 = reader2.load_data(H5TEST2)
assert np.allclose(data2['GHI'], H5TABLE['GlobalHorizontalRadiation'])
assert data1['GHI'].units == UREG('W/m**2')
assert np.allclose(data2['DNI'], H5TABLE['DirectNormalRadiation'])
assert data1['DNI'].units == UREG('W/m**2')
assert np.allclose(data2['Tdry'], H5TABLE['DryBulbTemperature'])
assert data1['Tdry'].units == UREG.degC
teardown_hdf5_test_data()
return reader1, data1, reader2, data2
def wrapper(*params, **kw):
vals = []
for p, a in zip(params, args):
if a is None:
vals.append(p)
LOGGER.info('Skipping param with no units')
continue
vals.append(p.to(UREG(a)).magnitude)
LOGGER.info('hey I am inside the wrapper!')
LOGGER.debug(f)
LOGGER.debug(ret)
LOGGER.debug(args)
return [rv * UREG(r) for rv, r in zip(f(*vals, **kw), ret)]
def _apply_units(data_data, data_units, fname):
"""
Apply units to data.
:param data_data: NumPy structured array with data from fname.
:type data_data: :class:`numpy.ndarray`
:param data_units: Units of fields in data_data.
:type data_units: dict
:param fname: Name of file from which data_data was read.
:type fname: str
:returns: Dictionary of data with units applied.
:rtype: dict
:raises: :exc:`~simkit.core.exceptions.UnnamedDataError`
"""
data_names = data_data.dtype.names
# raise error if NumPy data doesn't have names
if not data_names:
raise UnnamedDataError(fname)
data = dict.fromkeys(data_names) # dictionary of data read by NumPy
# iterate over data read by NumPy
for data_name in data_names:
if data_name in data_units:
# if units specified in parameters, then convert to string
units = str(data_units[data_name])
data[data_name] = data_data[data_name] * UREG(units)
elif np.issubdtype(data_data[data_name].dtype, str):
# if no units specified and is string
data[data_name] = data_data[data_name].tolist()
else:
data[data_name] = data_data[data_name]
return data
def __init__(self):
#: outputs initial value
self.initial_value = {}
#: size of outputs
self.size = {}
#: outputs uncertainty
self.uncertainty = {}
#: variance
self.variance = {}
#: jacobian
self.jacobian = {}
#: outputs isconstant flag
self.isconstant = {}
#: outputs isproperty flag
self.isproperty = {}
#: name of corresponding time series, ``None`` if no time series
self.timeseries = {}
#: name of :class:`Output` superclass
self.output_source = {}
#: calculation outputs
self.outputs = {}
for k, v in self.parameters.items():
self.initial_value[k] = v.get('init') # returns None if missing
self.size[k] = v.get('size') or 1 # minimum size is 1
self.uncertainty[k] = None # uncertainty for outputs is calculated
self.isconstant[k] = v.get('isconstant', False) # True or False
self.isproperty[k] = v.get('isproperty', False) # True or False
units = str(v.get('units', '')) # default is non-dimensional
# NOTE: np.empty is faster than zeros!
self.outputs[k] = Q_(np.zeros((1, self.size[k])), UREG(units))
# NOTE: Initial values are assigned and outputs resized when
# simulation "start" method is called from the model.
self.timeseries[k] = v.get('timeseries') # None if not time series
self.output_source[k] = self.__class__.__name__ # output source
def __init__(self):
meta = getattr(self, CalcBase._meta_attr)
parameters = getattr(self, CalcBase._param_attr)
#: ``True`` if always calculated (day and night)
self.always_calc = dict.fromkeys(parameters,
getattr(meta, 'always_calc', False))
freq = getattr(meta, 'frequency', [1, ''])
#: frequency calculation is calculated in intervals or units of time
self.frequency = dict.fromkeys(parameters,
freq[0] * UREG(str(freq[1])))
#: dependencies
self.dependencies = dict.fromkeys(parameters,
getattr(meta, 'dependencies', []))
#: name of :class:`Calc` superclass
self.calc_source = dict.fromkeys(parameters, self.__class__.__name__)
#: calculator
self.calculator = dict.fromkeys(
parameters, getattr(meta, 'calculator', Calculator))
#: ``True`` if calculations are dynamic, ``False`` if static
self.is_dynamic = dict.fromkeys(parameters,
getattr(meta, 'is_dynamic', False))
#: calculations
self.calcs = {}
for k, v in parameters.iteritems():
self.calcs[k] = {
key: v[key]
for key in ('formula', 'args', 'returns')
}
keys = ('dependencies', 'always_calc', 'frequency', 'calculator',
'is_dynamic')
for key in keys:
value = v.get(key)
if value is not None:
getattr(self, key)[k] = value
def load_data(self, filename, *args, **kwargs):
"""
Load text data from different sheets.
"""
# load text data
data = super(MixedTextXLS, self).load_data(filename)
# iterate through sheets in parameters
for sheet_params in self.parameters.itervalues():
# iterate through the parameters on each sheet
for param, pval in sheet_params.iteritems():
pattern = pval.get('pattern', EFG_PATTERN) # get pattern
re_meth = pval.get('method', 'search') # get re method
# whitelist re methods, getattr could be considered harmful
if re_meth in RE_METH:
re_meth = getattr(re, pval.get('method', 'search'))
else:
msg = 'Only', '"%s", ' * len(RE_METH) % tuple(RE_METH)
msg += 'regex methods are allowed.'
raise AttributeError(msg)
# if not isinstance(data[param], basestring):
# re_meth = lambda p, dp: [re_meth(p, d) for d in dp]
match = re_meth(pattern, data[param]) # get matches
if match:
try:
match = match.groups()
except AttributeError:
match = [m.groups() for m in match]
npdata = np.array(match, dtype=float).squeeze()
data[param] = npdata * UREG(str(pval.get('units') or ''))
else:
raise MixedTextNoMatchError(re_meth, pattern, data[param])
return data
def register(self, newdata, *args, **kwargs):
"""
Register data in registry. Meta for each data is specified by positional
or keyword arguments after the new data and consists of the following:
* ``uncertainty`` - Map of uncertainties in percent corresponding to new
keys. The uncertainty keys must be a subset of the new data keys.
* ``variance`` - Square of the uncertainty (no units).
* ``isconstant``: Map corresponding to new keys whose values are``True``
if constant or ``False`` if periodic. These keys must be a subset of
the new data keys.
* ``timeseries``: Name of corresponding time series data, ``None`` if no
time series. _EG_: DNI data ``timeseries`` attribute might be set to a
date/time data that it corresponds to. More than one data can have the
same ``timeseries`` data.
* ``data_source``: the
:class:`~simkit.core.data_sources.DataSource` superclass that
was used to acquire this data. This can be used to group data from a
specific source together.
:param newdata: New data to add to registry. When registering new data,
keys are not allowed to override existing keys in the data
registry.
:type newdata: mapping
:raises:
:exc:`~simkit.core.exceptions.UncertaintyPercentUnitsError`
"""
kwargs.update(zip(self.meta_names, args))
# check uncertainty has units of percent
uncertainty = kwargs['uncertainty']
variance = kwargs['variance']
isconstant = kwargs['isconstant']
# check uncertainty is percent
if uncertainty:
for k0, d in uncertainty.items():
for k1, v01 in d.items():
units = v01.units
if units != UREG('percent'):
keys = '%s-%s' % (k0, k1)
raise UncertaintyPercentUnitsError(keys, units)
# check variance is square of uncertainty
if variance and uncertainty:
for k0, d in variance.items():
for k1, v01 in d.items():
keys = '%s-%s' % (k0, k1)
missing = k1 not in uncertainty[k0]
v2 = np.asarray(uncertainty[k0][k1].to('fraction').m) ** 2.0
if missing or not np.allclose(np.asarray(v01), v2):
raise UncertaintyVarianceError(keys, v01)
# check that isconstant is boolean
if isconstant:
for k, v in isconstant.items():
if not isinstance(v, bool):
classname = self.__class__.__name__
error_msg = ['%s meta "isconstant" should be' % classname,
'boolean, but it was "%s" for "%s".' % (v, k)]
raise TypeError(' '.join(error_msg))
# call super method, meta must be passed as kwargs!
super(DataRegistry, self).register(newdata, **kwargs)
def apply_units_to_cache(self, data):
"""
Apply units to :class:`ParameterizedXLS` data reader.
"""
# parameter
parameter_name = self.parameters['parameter']['name']
parameter_units = str(self.parameters['parameter']['units'])
data[parameter_name] *= UREG(parameter_units)
# data
self.parameters.pop('parameter')
return super(ParameterizedXLS, self).apply_units_to_cache(data)
def load_data(self, filename, *args, **kwargs):
"""
Load parameterized data from different sheets.
"""
# load parameterized data
data = super(ParameterizedXLS, self).load_data(filename)
# add parameter to data
parameter_name = self.parameterization['parameter']['name']
parameter_values = self.parameterization['parameter']['values']
parameter_units = str(self.parameterization['parameter']['units'])
data[parameter_name] = parameter_values * UREG(parameter_units)
# number of sheets
num_sheets = len(self.parameterization['parameter']['sheets'])
# parse and concatenate parameterized data
for key in self.parameterization['data']:
units = str(self.parameterization['data'][key].get('units')) or ''
datalist = []
for n in xrange(num_sheets):
k = key + '_' + str(n)
datalist.append(data[k].reshape((1, -1)))
data.pop(k) # remove unused data keys
data[key] = np.concatenate(datalist, axis=0) * UREG(units)
return data
def apply_units_to_cache(self, data):
"""
Apply units to cached data read using :class:`JSONReader`.
:param data: Cached data.
:type data: dict
:return: data with units
"""
# iterate through sheets in parameters
# iterate through the parameters on each sheet
for param, pval in self.parameters.iteritems():
# try to apply units
try:
data[param] *= UREG(str(pval.get('units') or ''))
except TypeError:
continue
return data
def _read_header(f, header_param):
"""
Read and parse data from 1st line of a file.
:param f: :func:`file` or :class:`~StringIO.StringIO` object from which to
read 1st line.
:type f: file
:param header_param: Parameters used to parse the data from the header.
Contains "delimiter" and "fields".
:type header_param: dict
:returns: Dictionary of data read from header.
:rtype: dict
:raises: :exc:`~simkit.core.exceptions.UnnamedDataError`
The **header_param** argument contains keys to read the 1st line of **f**.
If "delimiter" is ``None`` or missing, the default delimiter is a comma,
otherwise "delimiter" can be any single character, integer or sequence of
``int``.
* single character -- a delimiter
* single integer -- uniform fixed width
* sequence of ``int`` -- fixed widths, the number of fields should \
correspond to the length of the sequence.
The "fields" key is a list of (parameter-name, parameter-type[, parameter-
units]) lists.
"""
# default delimiter is a comma, can't be None
header_delim = str(header_param.get('delimiter', ','))
# don't allow unnamed fields
if 'fields' not in header_param:
raise UnnamedDataError(f.name)
header_fields = {field[0]: field[1:] for field in header_param['fields']}
# header_names can't be generator b/c DictReader needs list, and can't be
# dictionary b/c must be same order as 'fields' to match data readby csv
header_names = [field[0] for field in header_param['fields']]
# read header
header_str = StringIO(f.readline()) # read the 1st line
# use csv because it will preserve quoted fields with commas
# make a csv.DictReader from header string, use header names for
# fieldnames and set delimiter to header delimiter
header_reader = csv.DictReader(header_str,
header_names,
delimiter=header_delim,
skipinitialspace=True)
data = header_reader.next() # parse the header dictionary
# iterate over items in data
for k, v in data.iteritems():
header_type = header_fields[k][0] # spec'd type
# whitelist header types
if isinstance(header_type, basestring):
if header_type.lower().startswith('int'):
header_type = int # coerce to integer
elif header_type.lower().startswith('long'):
header_type = long # coerce to long integer
elif header_type.lower().startswith('float'):
header_type = float # to floating decimal point
elif header_type.lower().startswith('str'):
header_type = str # coerce to string
elif header_type.lower().startswith('bool'):
header_type = bool # coerce to boolean
else:
raise TypeError('"%s" is not a supported type.' % header_type)
# WARNING! Use of `eval` considered harmful. `header_type` is read
# from JSON file, not secure input, could be used to exploit system
data[k] = header_type(v) # cast v to type
# check for units in 3rd element
if len(header_fields[k]) > 1:
units = UREG(str(header_fields[k][1])) # spec'd units
data[k] = data[k] * units # apply units
return data
class Simulation(object):
"""
A class for simulations.
:param simfile: Filename of simulation configuration file.
:type simfile: str
:param settings: keyword name of simulation parameter to use for settings
:type str:
Simulation attributes can be passed directly as keyword arguments directly
to :class:`~simkit.core.simulations.Simulation` or in a JSON file or as
class attributes in a subclass or a combination of all 3 methods.
To get a list of :class:`~simkit.core.simulations.Simulation` attributes
and defaults get the :attr:`~simkit.core.simulations.Simulation.attrs`
attribute.
Any additional settings provided as keyword arguments will override settings
from file.
"""
__metaclass__ = SimBase
attrs = {
'ID': None,
'path': os.path.join('~', 'SimKit', 'Simulations'),
'commands': ['start', 'pause'],
'data': None,
'thresholds': None,
'interval': 1 * UREG.hour,
'sim_length': 1 * UREG.year,
'display_frequency': 1,
'display_fields': None,
'write_frequency': 8760,
'write_fields': None
}
deprecated = {
'interval': 'interval_length',
'sim_length': 'simulation_length'
}
def __init__(self, simfile=None, settings=None, **kwargs):
# load simfile if it's an argument
if simfile is not None:
# read and load JSON parameter map file as "parameters"
self.param_file = simfile
with open(self.param_file, 'r') as param_file:
file_params = json.load(param_file)
#: simulation parameters from file
self.parameters = {
settings: SimParameter(**params)
for settings, params in file_params.iteritems()
}
# if not subclassed and metaclass skipped, then use kwargs
if not hasattr(self, 'parameters'):
#: parameter file
self.param_file = None
#: simulation parameters from keyword arguments
self.parameters = kwargs
else:
# use first settings
if settings is None:
self.settings, self.parameters = self.parameters.items()[0]
else:
#: name of sim settings used for parameters
self.settings = settings
self.parameters = self.parameters[settings]
# use any keyword arguments instead of parameters
self.parameters.update(kwargs)
# make pycharm happy - attributes assigned in loop by attrs
self.thresholds = {}
self.display_frequency = 0
self.display_fields = {}
self.write_frequency = 0
self.write_fields = {}
# pop deprecated attribute names
for k, v in self.deprecated.iteritems():
val = self.parameters['extras'].pop(v, None)
# update parameters if deprecated attr used and no new attr
if val and k not in self.parameters:
self.parameters[k] = val
# Attributes
for k, v in self.attrs.iteritems():
setattr(self, k, self.parameters.get(k, v))
# member docstrings are in documentation since attrs are generated
if self.ID is None:
# generate id from object class name and datetime in ISO format
self.ID = id_maker(self)
if self.path is not None:
# expand environment variables, ~ and make absolute path
self.path = os.path.expandvars(os.path.expanduser(self.path))
self.path = os.path.abspath(self.path)
# convert simulation interval to Pint Quantity
if isinstance(self.interval, basestring):
self.interval = UREG(self.interval)
elif not isinstance(self.interval, Q_):
self.interval = self.interval[0] * UREG(str(self.interval[1]))
# convert simulation length to Pint Quantity
if isinstance(self.sim_length, basestring):
self.sim_length = UREG(self.sim_length)
elif not isinstance(self.sim_length, Q_):
self.sim_length = self.sim_length[0] * UREG(str(
self.sim_length[1]))
# convert simulation length to interval units to calc total intervals
sim_to_interval_units = self.sim_length.to(self.interval.units)
#: total number of intervals simulated
self.number_intervals = np.ceil(sim_to_interval_units / self.interval)
#: interval index, start at zero
self.interval_idx = 0
#: pause status
self._ispaused = False
#: finished status
self._iscomplete = False
#: initialized status
self._isinitialized = False
#: order of calculations
self.calc_order = []
#: command queue
self.cmd_queue = Queue.Queue()
#: index iterator
self.idx_iter = self.index_iterator()
#: data loaded status
self._is_data_loaded = False
@property
def ispaused(self):
"""
Pause property, read only. True if paused.
"""
return self._ispaused
@property
def iscomplete(self):
"""
Completion property, read only. True if finished.
"""
return self._iscomplete
@property
def isinitialized(self):
"""
Initialization property, read only. True if initialized.
"""
return self._isinitialized
@property
def is_data_loaded(self):
"""
Data loaded property, read only. True if data loaded.
"""
return self._is_data_loaded
def check_data(self, data):
"""
Check if data loaded for all sources in data layer.
:param data: data layer from model
:type data: :class:`~simkit.core.layer.Data`
:return: dictionary of data sources and objects or `None` if not loaded
"""
data_objs = {
data_src: data.objects.get(data_src)
for data_src in data.layer
}
self._is_data_loaded = all(data_objs.values())
return data_objs
def initialize(self, calc_reg):
"""
Initialize the simulation. Organize calculations by dependency.
:param calc_reg: Calculation registry.
:type calc_reg:
:class:`~simkit.core.calculation.CalcRegistry`
"""
self._isinitialized = True
# TODO: if calculations are edited, loaded, added, etc. then reset
self.calc_order = topological_sort(calc_reg.dependencies)
def index_iterator(self):
"""
Generator that resumes from same index, or restarts from sent index.
"""
idx = 0 # index
while idx < self.number_intervals:
new_idx = yield idx
idx += 1
if new_idx:
idx = new_idx - 1
# TODO: change start to run
def start(self, model, progress_hook=None):
"""
Start the simulation from time zero.
:param model: Model with layers and registries containing parameters
:type: :class:`~simkit.core.models.Model`
:param progress_hook: A function that receives either a string or a
list containing the index followed by tuples of the data or outputs
names and values specified by ``write_fields`` in the simfile.
:type progress_hook: function
The model registries should contain the following layer registries:
* :class:`~simkit.core.data_sources.DataRegistry`,
* :class:`~simkit.core.formulas.FormulaRegistry`,
* :class:`~simkit.core.outputs.OutputRegistry`,
* :class:`~simkit.core.calculation.CalcRegistry`
"""
# check if data loaded
data_objs = self.check_data(model.data)
if not self.is_data_loaded:
raise MissingDataError([ds for ds in data_objs if ds is None])
# get layer registries
data_reg = model.registries['data']
formula_reg = model.registries['formulas']
out_reg = model.registries['outputs']
calc_reg = model.registries['calculations']
# initialize
if not self.isinitialized:
self.initialize(calc_reg)
# default progress hook
if not progress_hook:
progress_hook = functools.partial(sim_progress_hook,
display_header=True)
# start, resume or restart
if self.ispaused:
# if paused, then resume, do not resize outputs again.
self._ispaused = False # change pause state
progress_hook('resume simulation')
elif self.iscomplete:
# if complete, then restart, do not resize outputs again.
self._iscomplete = False # change pause state
progress_hook('restart simulation')
self.idx_iter = self.index_iterator()
else:
# resize outputs
# assumes that self.write_frequency is immutable
# TODO: allow self.write_frequency to be changed
# only resize outputs first time simulation is started
# repeat output rows to self.write_frequency
# put initial conditions of outputs last so it's copied when
# idx == 0
progress_hook('resize outputs') # display progress
for k in out_reg:
if out_reg.isconstant[k]:
continue
# repeat rows (axis=0)
out_reg[k] = out_reg[k].repeat(self.write_frequency, 0)
_initial_value = out_reg.initial_value[k]
if not _initial_value:
continue
if isinstance(_initial_value, basestring):
# initial value is from data registry
# assign in a scalar to a vector fills in the vector, yes!
out_reg[k][-1] = data_reg[_initial_value]
else:
out_reg[k][-1] = _initial_value * out_reg[k].units
progress_hook('start simulation')
# check and/or make SimKit_Simulations and simulation ID folders
mkdir_p(self.path)
sim_id_path = os.path.join(self.path, self.ID)
mkdir_p(sim_id_path)
# header & units for save files
data_fields = self.write_fields.get('data', []) # any data fields
out_fields = self.write_fields.get('outputs', []) # any outputs fields
save_header = tuple(data_fields + out_fields) # concatenate fields
# get units as strings from data & outputs
data_units = [str(data_reg[f].dimensionality) for f in data_fields]
out_units = [str(out_reg[f].dimensionality) for f in out_fields]
save_units = tuple(data_units + out_units) # concatenate units
# string format for header & units
save_str = ('%s' + ',%s' * (len(save_header) - 1)) + '\n' # format
save_header = (save_str * 2) % (save_header + save_units) # header
save_header = save_header[:-1] # remove trailing new line
# ===================
# Static calculations
# ===================
progress_hook('static calcs')
for calc in self.calc_order:
if not calc_reg.is_dynamic[calc]:
calc_reg.calculator[calc].calculate(calc_reg[calc],
formula_reg, data_reg,
out_reg)
# ====================
# Dynamic calculations
# ====================
progress_hook('dynamic calcs')
# TODO: assumes that interval size and indices are same, but should
# interpolate for any size interval or indices
for idx_tot in self.idx_iter:
self.interval_idx = idx_tot # update simulation interval counter
idx = idx_tot % self.write_frequency
# update properties
for k, v in out_reg.isproperty.iteritems():
# set properties from previous interval at night
if v:
out_reg[k][idx] = out_reg[k][idx - 1]
# night if any threshold exceeded
if self.thresholds:
night = not all(
limits[0] < data_reg[data][idx] < limits[1]
for data, limits in self.thresholds.iteritems())
else:
night = None
# daytime or always calculated outputs
for calc in self.calc_order:
# Determine if calculation is scheduled for this timestep
# TODO: add ``start_at`` parameter combined with ``frequency``
freq = calc_reg.frequency[calc]
if not freq.dimensionality:
is_scheduled = (idx_tot % freq) == 0
else:
# Frequency with units of time
is_scheduled = ((idx_tot * self.interval) % freq) == 0
is_scheduled = (is_scheduled
and (not night or calc_reg.always_calc[calc]))
if calc_reg.is_dynamic[calc] and is_scheduled:
calc_reg.calculator[calc].calculate(calc_reg[calc],
formula_reg,
data_reg,
out_reg,
timestep=self.interval,
idx=idx)
# display progress
if not (idx % self.display_frequency):
progress_hook(self.format_progress(idx, data_reg, out_reg))
# disp_head = False
# create an index for the save file, 0 if not saving
if not ((idx_tot + 1) % self.write_frequency):
savenum = (idx_tot + 1) / self.write_frequency
elif idx_tot == self.number_intervals - 1:
# save file index should be integer!
savenum = int(
np.ceil((idx_tot + 1) / float(self.write_frequency)))
else:
savenum = 0 # not saving this iteration
# save file to disk
if savenum:
savename = self.ID + '_' + str(savenum) + '.csv' # filename
savepath = os.path.join(sim_id_path, savename) # path
# create array of all data & outputs to save
save_array = self.format_write(data_reg, out_reg, idx + 1)
# save as csv using default format & turn comments off
np.savetxt(savepath,
save_array,
delimiter=',',
header=save_header,
comments='')
try:
cmd = self.cmd_queue.get_nowait()
except Queue.Empty:
continue
if cmd == 'pause':
self._ispaused = True
return
self._iscomplete = True # change completion status
def format_progress(self, idx, data_reg, out_reg):
data_fields = self.display_fields.get('data', []) # data fields
data_args = [(f, data_reg[f][idx]) for f in data_fields]
out_fields = self.display_fields.get('outputs', []) # outputs fields
out_args = [(f, out_reg[f][idx]) for f in out_fields]
return [idx] + data_args + out_args
def format_write(self, data_reg, out_reg, idx=None):
data_fields = self.write_fields.get('data', []) # any data fields
data_args = [data_reg[f][:idx].reshape((-1, 1)) for f in data_fields]
out_fields = self.write_fields.get('outputs', []) # any outputs fields
out_args = [out_reg[f][:idx] for f in out_fields]
return np.concatenate(data_args + out_args, axis=1)
def pause(self, progress_hook=None):
"""
Pause the simulation. How is this different from stopping it? Maintain
info sufficient to restart simulation. Sets ``is_paused`` to True.
Will this state allow analysis? changing parameters? What can you do
with a paused simulation?
Should be capable of saving paused simulation for loading/resuming
later, that is the main usage. EG: someone else need computer, or power
goes out, so on battery backup quickly pause simulation, and save.
Is save automatic? Should there be a parameter for auto save changed?
"""
# default progress hook
if progress_hook is None:
progress_hook = sim_progress_hook
progress_hook('simulation paused')
self.cmd_queue.put('pause')
self._ispaused = True
def load(self, model, progress_hook=None, *args, **kwargs):
# default progress hook
if progress_hook is None:
progress_hook = sim_progress_hook
data = kwargs.get('data', {})
if not data and args:
data = args[0]
for k, v in data.iteritems():
progress_hook('loading simulation for %s' % k)
model.data.open(k, **v)
self.check_data(model.data)
def run(self, model, progress_hook=None, *args, **kwargs):
# default progress hook
if progress_hook is None:
progress_hook = sim_progress_hook
progress_hook('running simulation')
self.load(model, progress_hook, *args, **kwargs)
self.start(model, progress_hook)
def index_registry(args, reg, ts=None, idx=None):
"""
Index into a :class:`~simkit.core.Registry` to return arguments
from :class:`~simkit.core.data_sources.DataRegistry` and
:class:`~simkit.core.outputs.OutputRegistry` based on the
calculation parameter file.
:param args: Arguments field from the calculation parameter file.
:param reg: Registry in which to index to get the arguments.
:type reg: :class:`~simkit.core.data_sources.DataRegistry`,
:class:`~simkit.core.outputs.OutputRegistry`
:param ts: Time step [units of time].
:param idx: [None] Index of current time step for dynamic calculations.
Required arguments for static and dynamic calculations are specified in the
calculation parameter file by the "args" key. Arguments can be from
either the data registry or the outputs registry, which is denoted by the
"data" and "outputs" keys. Each argument is a dictionary whose key is the
name of the argument in the formula specified and whose value can be one of
the following:
* The name of the argument in the registry ::
{"args": {"outputs": {"T_bypass": "******"}}}
maps the formula argument "T_bypass" to the outputs registry item
"T_bypass_diode".
* A list with the name of the argument in the registry as the first element
and a negative integer denoting the index relative to the current
timestep as the second element ::
{"args": {"data": {"T_cell": ["Tcell", -1]}}}
indexes the previous timestep of "Tcell" from the data registry.
* A list with the name of the argument in the registry as the first element
and a list of positive integers denoting the index into the item from the
registry as the second element ::
{"args": {"data": {"cov": ["bypass_diode_covariance", [2]]}}}
indexes the third element of "bypass_diode_covariance".
* A list with the name of the argument in the registry as the first
element, a negative real number denoting the time relative to the current
timestep as the second element, and the units of the time as the third ::
{"args": {"data": {"T_cell": ["Tcell", -1, 'day']}}}
indexes the entire previous day of "Tcell".
"""
# TODO: move this to new Registry method or __getitem__
# TODO: replace idx with datetime object and use timeseries to interpolate
# into data, not necessary for outputs since that will conform to idx
rargs = dict.fromkeys(args) # make dictionary from arguments
# iterate over arguments
for k, v in args.iteritems():
# var ------------------ states ------------------
# idx ===== not None ===== ======= None =======
# isconstant True False None True False None
# is_dynamic no yes yes no no no
is_dynamic = idx and not reg.isconstant.get(v)
# switch based on string type instead of sequence
if isinstance(v, basestring):
# the default assumes the current index
rargs[k] = reg[v][idx] if is_dynamic else reg[v]
elif len(v) < 3:
if reg.isconstant[v[0]]:
# only get indices specified by v[1]
# tuples interpreted as a list of indices, see
# NumPy basic indexing: Dealing with variable
# numbers of indices within programs
rargs[k] = reg[v[0]][tuple(v[1])]
elif v[1] < 0:
# specified offset from current index
rargs[k] = reg[v[0]][idx + v[1]]
else:
# get indices specified by v[1] at current index
rargs[k] = reg[v[0]][idx][tuple(v[1])]
else:
# specified timedelta from current index
dt = 1 + (v[1] * UREG(str(v[2])) / ts).item()
# TODO: deal with fractions of timestep
rargs[k] = reg[v[0]][(idx + dt):(idx + 1)]
return rargs
its units are, what the data will be called in calculations and any other
meta-data the registry requires.
"""
from simkit.core import (UREG, Registry, SimKitJSONEncoder, CommonBase,
Parameter)
from simkit.core.data_readers import JSONReader
from simkit.core.exceptions import (UncertaintyPercentUnitsError,
UncertaintyVarianceError)
import json
import os
import time
from copy import copy
import numpy as np
DFLT_UNC = 1.0 * UREG('percent') # default uncertainty
class DataParameter(Parameter):
"""
Field for data parameters.
"""
_attrs = ['units', 'uncertainty', 'isconstant', 'timeseries']
class DataRegistry(Registry):
"""
A registry for data sources. The meta names are: ``uncertainty``,
``variance``, ``isconstant``, ``timeseries`` and ``data_source``
"""
#: meta names
def load_data(self, filename, *args, **kwargs):
"""
Load parameters from Excel spreadsheet.
:param filename: Name of Excel workbook with data.
:type filename: str
:returns: Data read from Excel workbook.
:rtype: dict
"""
# workbook read from file
workbook = open_workbook(filename, verbosity=True)
data = {} # an empty dictionary to store data
# iterate through sheets in parameters
# iterate through the parameters on each sheet
for param, pval in self.parameters.iteritems():
sheet = pval['extras']['sheet']
# get each worksheet from the workbook
worksheet = workbook.sheet_by_name(sheet)
# split the parameter's range elements
prng0, prng1 = pval['extras']['range']
# missing "units", json ``null`` and Python ``None`` all OK!
# convert to str from unicode, None to '' (dimensionless)
punits = str(pval.get('units') or '')
# replace None with empty list
if prng0 is None:
prng0 = []
if prng1 is None:
prng1 = []
# FIXME: Use duck-typing here instead of type-checking!
# if both elements in range are `int` then parameter is a cell
if isinstance(prng0, int) and isinstance(prng1, int):
datum = worksheet.cell_value(prng0, prng1)
# if the either element is a `list` then parameter is a slice
elif isinstance(prng0, list) and isinstance(prng1, int):
datum = worksheet.col_values(prng1, *prng0)
elif isinstance(prng0, int) and isinstance(prng1, list):
datum = worksheet.row_values(prng0, *prng1)
# if both elements are `list` then parameter is 2-D
else:
datum = []
for col in xrange(prng0[1], prng1[1]):
datum.append(worksheet.col_values(col, prng0[0], prng1[0]))
# duck typing that datum is real
try:
npdatum = np.array(datum, dtype=np.float)
except ValueError as err:
# check for iterable:
# if `datum` can't be coerced to float, then it must be
# *string* & strings *are* iterables, so don't check!
# check for strings:
# data must be real or *all* strings!
# empty string, None or JSON null also OK
# all([]) == True but any([]) == False
if not datum:
data[param] = None # convert empty to None
elif all(isinstance(_, basestring) for _ in datum):
data[param] = datum # all str is OK (EG all 'TMY')
elif all(not _ for _ in datum):
data[param] = None # convert list of empty to None
else:
raise err # raise ValueError if not all real or str
else:
data[param] = npdatum * UREG(punits)
# FYI: only put one statement into try-except test otherwise
# might catch different error than expected. use ``else`` as
# option to execute only if exception *not* raised.
return data
def __init__(self, simfile=None, settings=None, **kwargs):
# load simfile if it's an argument
if simfile is not None:
# read and load JSON parameter map file as "parameters"
self.param_file = simfile
with open(self.param_file, 'r') as param_file:
file_params = json.load(param_file)
#: simulation parameters from file
self.parameters = {
settings: SimParameter(**params)
for settings, params in file_params.iteritems()
}
# if not subclassed and metaclass skipped, then use kwargs
if not hasattr(self, 'parameters'):
#: parameter file
self.param_file = None
#: simulation parameters from keyword arguments
self.parameters = kwargs
else:
# use first settings
if settings is None:
self.settings, self.parameters = self.parameters.items()[0]
else:
#: name of sim settings used for parameters
self.settings = settings
self.parameters = self.parameters[settings]
# use any keyword arguments instead of parameters
self.parameters.update(kwargs)
# make pycharm happy - attributes assigned in loop by attrs
self.thresholds = {}
self.display_frequency = 0
self.display_fields = {}
self.write_frequency = 0
self.write_fields = {}
# pop deprecated attribute names
for k, v in self.deprecated.iteritems():
val = self.parameters['extras'].pop(v, None)
# update parameters if deprecated attr used and no new attr
if val and k not in self.parameters:
self.parameters[k] = val
# Attributes
for k, v in self.attrs.iteritems():
setattr(self, k, self.parameters.get(k, v))
# member docstrings are in documentation since attrs are generated
if self.ID is None:
# generate id from object class name and datetime in ISO format
self.ID = id_maker(self)
if self.path is not None:
# expand environment variables, ~ and make absolute path
self.path = os.path.expandvars(os.path.expanduser(self.path))
self.path = os.path.abspath(self.path)
# convert simulation interval to Pint Quantity
if isinstance(self.interval, basestring):
self.interval = UREG(self.interval)
elif not isinstance(self.interval, Q_):
self.interval = self.interval[0] * UREG(str(self.interval[1]))
# convert simulation length to Pint Quantity
if isinstance(self.sim_length, basestring):
self.sim_length = UREG(self.sim_length)
elif not isinstance(self.sim_length, Q_):
self.sim_length = self.sim_length[0] * UREG(str(
self.sim_length[1]))
# convert simulation length to interval units to calc total intervals
sim_to_interval_units = self.sim_length.to(self.interval.units)
#: total number of intervals simulated
self.number_intervals = np.ceil(sim_to_interval_units / self.interval)
#: interval index, start at zero
self.interval_idx = 0
#: pause status
self._ispaused = False
#: finished status
self._iscomplete = False
#: initialized status
self._isinitialized = False
#: order of calculations
self.calc_order = []
#: command queue
self.cmd_queue = Queue.Queue()
#: index iterator
self.idx_iter = self.index_iterator()
#: data loaded status
self._is_data_loaded = False
def __init__(self):
# check for path listed in param file
path = getattr(self._meta, 'path', None)
if path is None:
proxy_file = self.param_file if self.param_file else __file__
# use the same path as the param file or this file if no param file
self._meta.path = os.path.dirname(proxy_file)
# check for path listed in param file
formula_importer = getattr(self._meta, 'formula_importer', None)
if formula_importer is None:
#: formula importer class, default is ``PyModuleImporter``
self._meta.formula_importer = PyModuleImporter
meta = getattr(self, '_meta', None) # options for formulas
importer_instance = self._meta.formula_importer(self.parameters, meta)
#: formulas loaded by the importer using specified parameters
self.formulas = importer_instance.import_formulas()
#: linearity determined by each data source?
self.islinear = {}
#: positional arguments
self.args = {}
#: expected units of returns and arguments as pair of tuples
self.units = {}
#: constant arguments that are not included in covariance calculation
self.isconstant = {}
# sequence of formulas, don't propagate uncertainty or units
for f in self.formulas:
self.islinear[f] = True
self.args[f] = inspect.getargspec(self.formulas[f]).args
formula_param = self.parameters # formulas key
# if formulas is a list or if it can't be iterated as a dictionary
# then log warning and return
try:
formula_param_generator = formula_param.iteritems()
except AttributeError as err:
LOGGER.warning('Attribute Error: %s', err.message)
return
# formula dictionary
for k, v in formula_param_generator:
if not v:
# skip formula if attributes are null or empty
continue
# get islinear formula attribute
is_linear = v.get('islinear')
if is_linear is not None:
self.islinear[k] = is_linear
# get positional arguments
f_args = v.get('args')
if f_args is not None:
self.args[k] = f_args
# get constant arguments to exclude from covariance
self.isconstant[k] = v.get('isconstant')
if self.isconstant[k] is not None:
argn = [
n for n, a in enumerate(self.args[k])
if a not in self.isconstant[k]
]
LOGGER.debug('%s arg nums: %r', k, argn)
self.formulas[k] = unc_wrapper_args(*argn)(self.formulas[k])
# get units of returns and arguments
self.units[k] = v.get('units')
if self.units[k] is not None:
# append units for covariance and Jacobian if all args
# constant and more than one return output
if self.isconstant[k] is not None:
# check if retval units is a string or None before adding
# extra units for Jacobian and covariance
ret_units = self.units[k][0]
if isinstance(ret_units, basestring) or ret_units is None:
self.units[k][0] = [ret_units]
try:
self.units[k][0] += [None, None]
except TypeError:
self.units[k][0] += (None, None)
# wrap function with Pint's unit wrapper
self.formulas[k] = UREG.wraps(*self.units[k])(self.formulas[k])